package com.example.sparkdemo.sql

import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Row, SparkSession}


/**
 *
 *
 * @author zhang.t.c
 * @date 2021/5/13
 */
object Spark2 {

  def main(args: Array[String]): Unit = {

    val sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparkSQL")
    val spark = SparkSession.builder()
      .config(sparkConf)
      .getOrCreate()

    import spark.implicits._


    // DataFrame
    val df1 = spark.read.json("datas/user.json")
    df1.show()

    df1.createOrReplaceGlobalTempView("user")

    spark.sql("select * from user").show()
    spark.sql("select name,age from user").show()
    spark.sql("select avg(age) from user").show()

    df1.select("name", "age").show()
    df1.select($"name", $"age" + 1).show()
    df1.select('name, 'age + 1).show()

    // DataSet
    val seq = Seq(1, 2, 3, 4)
    val ds1 = seq.toDS()
    df1.show()

    // RDD <=> DataFrame
    val rdd = spark.sparkContext.makeRDD(List((1, "a", 20), (2, "b", 30)))
    val df2 = rdd.toDF("id", "name", "age")
    val rdd1: RDD[Row] = df2.rdd


    // RDD <=> DataSet
    val ds3 = rdd.map {
      case (id, name, age) => {
        User(id, name, age)
      }
    }.toDS()
    val rdd2 = ds3.rdd

    // DataSet <=> DataFrame
    val ds2 = df2.as[User]
    val df3 = ds2.toDF()


    //udf
    spark.udf.register("prefix", (name: String) => {
      "Name:" + name
    })
    spark.sql("select prefix(name) from user").show()










    spark.stop()


  }

  case class User(id: Int, name: String, age: Int)

}
